# Updated by Wei on 2014/12/28 at school
# continued to work on Juan's BPP model
# Updated by Wei on 2014/12/03 at school
# Updated by Wei on 2014/11/22 night at school
# Updated by Wei on 2014/11/16 afternoon at school
# redirect all the file pointers after the elimination
# Updated by Wei on 2014/11/13 night at school
# Updated by Wei on 2014/11/10 night at school
# Updated by Wei on 2014/11/08
# Another attempt for Juan's BPP program
# Updated by Wei on 2014/11/05
# do it on the conference SIGSPATIAL 2014
# Updated by Wei on 2014/10/27 
# let's test whatever I can to do the 3D table
# Updated by Wei on 2014/09/30
# Updated by Wei on 2014/09/27
# Updated by Wei on 2014/09/07
# Updated by Wei on 2014/08/23
# build the fall through computation for each query
# Updated by Wei on 2014/07/24
# Try Juan's thing
# Updated by Wei on 2014/07/20
# -->rerun our baseline static unigram method for gov2
# -->rerun for clueweb09B, dynamic weight set to be 0,3 and 5
# -->test the gov2, dynamic weight set to be 5 to find that it is better than static.
# test Juan's 4 files:
# bigrams-0.75-qb-gov.static
# bigrams-0.75-qb-gov.dyn
# bigrams-1-qb.dyn
# bigrams-0.54-qb.dyn
# ready to make for the (pow to dot7) and (pow to 0) case 
# Updated by Wei on 2014/07/10
# what about exploring the relationship between the query processing cost and quality? 
# Updated by Wei on 2014/07/09
# This should be the final evaluation place.
# report both the quality(overlap, P@10) and QPC(model1,model2)
# for OR settings
# configuration steps
# (1) set NUM_OF_TOP_RESULTS_CUTOFF
# (2) set inputFileName2
# (3) set inputFileName1
# (4) set inputFileName0
# (5) set TOTAL_NUM_OF_POSTINGS
# (6) set numOfPostingNeededToBePoppedAtDebugPercentage

from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import gc
import math
import os
import random
import sys
import time
from sets import Set
import numpy as np

# print math.log(4,2)
# exit(1)

def getUnprunedDocumentResultsForQueries(inputFileName1, \
                                         NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE, \
                                         termANDTermIDDict, \
                                         qIDsWithTopKDocumentResultImpactDict, \
                                         top10PostingWithDocumentResultDict, \
                                         top10DocumentResultDict, \
                                         topKPostingWithDocumentResultDict, \
                                         NUM_OF_TOP_RESULTS_CUTOFF, \
                                         qidsToEvaluateDict, \
                                         qIDsWithTopKDocumentResultNumOfPostingsContributedDict, \
                                         qIDWithQueryTermsDict):
    numOfQueriesHavingQID = 0
    numOfQueriesHavingSearchContent = 0
    numOfResultsForAllTheQueries = 0
    
    inputFileHandler1 = open(inputFileName1,"r")
    currentLine = inputFileHandler1.readline()
    nextLine = ""
    currentQID = ""
    
    processFlag = False
    while currentLine:
        # sampling parsing line:
        # qid: 701
        if currentLine.startswith("qid:"):
            currentQID = currentLine.strip().split(":")[1].strip()
            print "currentQID:",currentQID
            print "numOfQueriesHavingQID:",numOfQueriesHavingQID
            numOfQueriesHavingQID += 1
            # debug
            #if numOfQueriesHavingQID == 500:
            #    break
            
            if currentQID in qidsToEvaluateDict:
                processFlag = True
            else:
                processFlag = False
        # sample parsing line:
        # Search: u s  oil industry history
        if currentLine.startswith("Search:") and processFlag:
            elementList = currentLine.strip().split(" ")
            
            currentSearchContent = ""
            
            queryTermList = []
            for element in elementList[1:]:
                if element.strip() != "":
                    queryTermList.append( element.strip() )        
            currentQueryLength = len(queryTermList)
            print "currentQueryLength:",currentQueryLength
    
            for element in queryTermList:
                currentSearchContent += element + " "
            
            print currentSearchContent
            numOfQueriesHavingSearchContent += 1
            
            
            currentQueryTermIndexDict = {}
            nextLine = inputFileHandler1.readline()
            # print nextLine.strip()
            parsedFlag = True
            
            if nextLine.strip().endswith("is NOT in the lexicon."):
                parsedFlag = False
            
            if nextLine.strip() != "" and parsedFlag:
                
                # skip the following sampling line: 
                # [2013-05-24 14:20:44] 38871 terms with their lists length have been loaded.
                # nextLine = inputFileHandler1.readline()
                
                nextLineElements = nextLine.strip().split(" ")
                # print "nextLineElements:",nextLineElements
                
                # sample parsing line: 
                # oil:0 industry:1 history:2 u:3 s:4
                for element in nextLineElements:
                    if element.split(":")[0] not in currentQueryTermIndexDict:
                        currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
                
                print "currentQueryTermIndexDict:",currentQueryTermIndexDict
                # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
                
                if currentQID not in qIDWithQueryTermsDict:
                    qIDWithQueryTermsDict[currentQID] = []
                
                for i in range(0,len(currentQueryTermIndexDict)):
                    currentTerm = currentQueryTermIndexDict[i]
                    qIDWithQueryTermsDict[currentQID].append(currentTerm)
                    
                # Now, it is time to read the threshold line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                    
                # Now, it is time to read computation method line
                currentLine = inputFileHandler1.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                
                # skip one line
                currentLine = inputFileHandler1.readline()
                
                # Now, it is time to read the actual training example line
                currentLine = inputFileHandler1.readline()
                numOfResultsForTheCurrentQuery = 0
                
                # for IMPORTANT DEBUG ONLY
                #print "len( currentLine.strip().split(" ") ):",len( currentLine.strip().split(" ") )
                #print "currentLine:",currentLine
                #exit(1)
                
                lineElements = currentLine.strip().split(" ")
                while len( lineElements ) == 25:
                    theRank = int(lineElements[0])
                    theDocID = lineElements[-3]
                    theScore = float(lineElements[-4])
                    documentResultKey = currentQID + "_" + theDocID
                    
                    if theRank <= NUM_OF_TOP_RESULTS_CUTOFF:
                        if currentQID not in qIDsWithTopKDocumentResultImpactDict:
                            qIDsWithTopKDocumentResultImpactDict[currentQID] = {}
                            qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID] = {}
                        if documentResultKey not in qIDsWithTopKDocumentResultImpactDict:
                            qIDsWithTopKDocumentResultImpactDict[currentQID][documentResultKey] = 0.0
                            qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID][documentResultKey] = 0
                        else:
                            print "duplicated document result."
                            print "documentResultKey:",documentResultKey
                            exit(1)

                    if theRank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:
                        top10DocumentResultDict[documentResultKey] = 1;
                    
                    upperBound = 0
                    if len(currentQueryTermIndexDict) >= 10:
                        upperBound = 10
                    else:
                        upperBound = len(currentQueryTermIndexDict)
                    
                    for i in range(0,upperBound):
                        currentTerm = currentQueryTermIndexDict[i]
                        if currentTerm not in termANDTermIDDict:
                            print currentTerm,"is NOT in the dict."
                            exit(1)
                        else:
                            currentTermID = termANDTermIDDict[currentTerm]
                        currentTermScore = float( lineElements[11 + i] )
                        if currentTermScore != 0.0:
                            postingKey = currentTermID + "_" + theDocID
                            
                            if theRank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:
                                if postingKey not in top10PostingWithDocumentResultDict:
                                    top10PostingWithDocumentResultDict[postingKey] = []
                                top10PostingWithDocumentResultDict[postingKey].append(documentResultKey)                            
                            
                            if theRank <= NUM_OF_TOP_RESULTS_CUTOFF:
                                if postingKey not in topKPostingWithDocumentResultDict:
                                    topKPostingWithDocumentResultDict[postingKey] = []
                                topKPostingWithDocumentResultDict[postingKey].append(documentResultKey)                    
                    
                    numOfResultsForTheCurrentQuery += 1
                    
                    currentLine = inputFileHandler1.readline()
                    lineElements = currentLine.strip().split(" ")
                    
                numOfResultsForAllTheQueries += numOfResultsForTheCurrentQuery
                
                # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
                # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
                # This sampling method has been developed on 2013/01/09
                
                print "numOfResultsForCurrentQuery:",numOfResultsForTheCurrentQuery
                print
            
        currentLine = inputFileHandler1.readline()
    
    print "numOfQueriesHavingQID:",numOfQueriesHavingQID
    print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
    print "numOfResultsForAllTheQueries:",numOfResultsForAllTheQueries
    print

print "Program Begins..."
NUM_OF_TOP_RESULTS_CUTOFF = 100
NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE = 10
QUERY_LENGTH_TO_EVALUATE = 0
print "len(sys.argv):",len(sys.argv)
if len(sys.argv) != 3:
    print "Illegal # of arguments"
    print "Usage: python programName.py all_postings_being_popped_file QUERY_LENGTH_TO_EVALUATE"
    exit(1)
else:
    QUERY_LENGTH_TO_EVALUATE = int(sys.argv[2])

# variables inits
# key: termID
# value: # of posting in the list length
termIDANDCurrentListLengthDict = {}
# key: qID
# value: a list of topK document results
qIDsWithTopKDocumentResultImpactDict = {}
# key: qID
# value: a list of topK document results
qIDsWithTopKDocumentResultNumOfPostingsContributedDict = {}

# key: topK posting
# value: a list of document results this poting serves
topKPostingWithDocumentResultDict = {}
# key: document results for TOP10
# value: (currently NOT USED)
top10DocumentResultDict = {}
# key: top10 posting
# value: a list of document results this posting serves
top10PostingWithDocumentResultDict = {}
# key: qID
# value: a list of query terms formed the current query
qIDWithQueryTermsDict = {}
# key: qID needs to be consider
# value: (currently NOT used)
qidsToEvaluateDict = {}
# for vidaserver1
# gov2 tail5K testing queries
inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/data/GOV2/gov2_tail5K_queries_testing_withQueryLengthAdded"
# clueweb09B testing queries
# inputFileName2 = "/home/vgc/wei/workspace/NYU_IRTK/data/Clueweb09B/clueweb09B_queries_ALL_sortedByRandom_testing_withQueryLengthAdded"
inputFileHandler = open(inputFileName2,"r")
for line in inputFileHandler.readlines():
    lineElements = line.strip().split(" ")
    currentQueryLength = int(lineElements[0])
    if QUERY_LENGTH_TO_EVALUATE == 0:
        currentQID = line.strip().split(":")[0].strip().split(" ")[1]
        qidsToEvaluateDict[currentQID] = 1
    else:      
        if currentQueryLength == QUERY_LENGTH_TO_EVALUATE:
            currentQID = line.strip().split(":")[0].strip().split(" ")[1]
            qidsToEvaluateDict[currentQID] = 1
inputFileHandler.close()
print "len(qidsToEvaluateDict):",len(qidsToEvaluateDict)

# for gov2 testing queries
# for vidaserver1:
inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/results/GOV2/rawResults_50%_TOP1000_OR_20140126Night"
# for clueweb09B testing queries
# inputFileName1 = "/home/vgc/wei/workspace/NYU_IRTK/results/Clueweb09B/rawResults_clueweb09B_TOP1000_testingQueries_OR_20140705"

termANDTermIDDict = {}
termIDAndTermDict = {}
termIDANDUnprunedListLengthDict = {}
# for gov2
# for vidaserver1:
inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/GOV2/partOfLexiconTermsWithTermIDForTail5KQueries"
# for clueweb09B
# inputFileName0 = "/home/vgc/wei/workspace/NYU_IRTK/data/Clueweb09B/partOfLexiconTermsWithTermIDForTestingQueries_clueweb09B"
inputFileHandler = open(inputFileName0,"r")
currentLine = inputFileHandler.readline()
currentLineNum = 0
while currentLine:
    if currentLineNum % 1000000 == 0:
        print currentLineNum,"lines loaded."
    currentLineElements = currentLine.strip().split(" ")
    currentTermID = currentLineElements[0]
    currentTerm = currentLineElements[1]
    currentTermListLength = int(currentLineElements[2])
    termANDTermIDDict[currentTerm] = currentTermID
    termIDAndTermDict[currentTermID] = currentTerm
    termIDANDUnprunedListLengthDict[currentTermID] = currentTermListLength 
    currentLine = inputFileHandler.readline()
    currentLineNum += 1
print "len(termANDTermIDDict): ",len(termANDTermIDDict)
print "len(termIDAndTermDict): ",len(termIDAndTermDict)
print "len(termIDANDUnprunedListLengthDict): ",len(termIDANDUnprunedListLengthDict)
inputFileHandler.close()

getUnprunedDocumentResultsForQueries(inputFileName1,NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE,termANDTermIDDict,qIDsWithTopKDocumentResultImpactDict,top10PostingWithDocumentResultDict,top10DocumentResultDict,topKPostingWithDocumentResultDict,NUM_OF_TOP_RESULTS_CUTOFF,qidsToEvaluateDict,qIDsWithTopKDocumentResultNumOfPostingsContributedDict,qIDWithQueryTermsDict)

print "OVERALL:"
print "len(qidsToEvaluateDict):",len(qidsToEvaluateDict)
print "QUERY_LENGTH_TO_EVALUATE:",QUERY_LENGTH_TO_EVALUATE
NUM_OF_TOPK_DOCUMENT_RESULT = 0
for currentQID in qIDsWithTopKDocumentResultImpactDict:
    NUM_OF_TOPK_DOCUMENT_RESULT += len( qIDsWithTopKDocumentResultImpactDict[currentQID] )
print "NUM_OF_TOPK_DOCUMENT_RESULT:",NUM_OF_TOPK_DOCUMENT_RESULT
print "len(topKPostingWithDocumentResultDict):",len(topKPostingWithDocumentResultDict)
print "len(top10DocumentResultDict):",len(top10DocumentResultDict)
print "len(top10PostingWithDocumentResultDict):",len(top10PostingWithDocumentResultDict)
print "NUM_OF_TOP_RESULTS_CUTOFF:",NUM_OF_TOP_RESULTS_CUTOFF
print "NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:",NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE
#################################################################################################################
# for gov2
TOTAL_NUM_OF_POSTINGS = 6451948010
numOfPostingNeededToBePoppedAtDebugPercentage = 5000000
# clueweb09B
# TOTAL_NUM_OF_POSTINGS = 17075485964
# numOfPostingNeededToBePoppedAtDebugPercentage = 13660388

numOfPostingNeededToBePoppedAt1Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.01)
numOfPostingNeededToBePoppedAt2Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.02)
numOfPostingNeededToBePoppedAt3Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.03)
numOfPostingNeededToBePoppedAt4Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.04)
numOfPostingNeededToBePoppedAt5Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.05)
numOfPostingNeededToBePoppedAt6Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.06)
numOfPostingNeededToBePoppedAt7Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.07)
numOfPostingNeededToBePoppedAt8Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.08)
numOfPostingNeededToBePoppedAt9Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.09)
numOfPostingNeededToBePoppedAt10Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.1)
numOfPostingNeededToBePoppedAt15Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.15)
numOfPostingNeededToBePoppedAt20Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.2)
numOfPostingNeededToBePoppedAt30Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.3)
numOfPostingNeededToBePoppedAt40Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.4)
numOfPostingNeededToBePoppedAt50Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.5)
numOfPostingNeededToBePoppedAt60Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.6)
numOfPostingNeededToBePoppedAt70Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.7)
numOfPostingNeededToBePoppedAt80Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.8)
numOfPostingNeededToBePoppedAt90Percentage = int(TOTAL_NUM_OF_POSTINGS * 0.9)
numOfPostingNeededToBePoppedAt100Percentage = int(TOTAL_NUM_OF_POSTINGS * 1.0)

print "numOfPostingNeededToBePoppedAtDebugPercentage:",numOfPostingNeededToBePoppedAtDebugPercentage
print "numOfPostingNeededToBePoppedAt1Percentage:",numOfPostingNeededToBePoppedAt1Percentage
print "numOfPostingNeededToBePoppedAt2Percentage:",numOfPostingNeededToBePoppedAt2Percentage
print "numOfPostingNeededToBePoppedAt3Percentage:",numOfPostingNeededToBePoppedAt3Percentage
print "numOfPostingNeededToBePoppedAt4Percentage:",numOfPostingNeededToBePoppedAt4Percentage
print "numOfPostingNeededToBePoppedAt5Percentage:",numOfPostingNeededToBePoppedAt5Percentage
print "numOfPostingNeededToBePoppedAt6Percentage:",numOfPostingNeededToBePoppedAt6Percentage
print "numOfPostingNeededToBePoppedAt7Percentage:",numOfPostingNeededToBePoppedAt7Percentage
print "numOfPostingNeededToBePoppedAt8Percentage:",numOfPostingNeededToBePoppedAt8Percentage
print "numOfPostingNeededToBePoppedAt9Percentage:",numOfPostingNeededToBePoppedAt9Percentage
print "numOfPostingNeededToBePoppedAt10Percentage:",numOfPostingNeededToBePoppedAt10Percentage
print "numOfPostingNeededToBePoppedAt15Percentage:",numOfPostingNeededToBePoppedAt15Percentage
print "numOfPostingNeededToBePoppedAt20Percentage:",numOfPostingNeededToBePoppedAt20Percentage
print "numOfPostingNeededToBePoppedAt30Percentage:",numOfPostingNeededToBePoppedAt30Percentage
print "numOfPostingNeededToBePoppedAt40Percentage:",numOfPostingNeededToBePoppedAt40Percentage
print "numOfPostingNeededToBePoppedAt50Percentage:",numOfPostingNeededToBePoppedAt50Percentage
print "numOfPostingNeededToBePoppedAt60Percentage:",numOfPostingNeededToBePoppedAt60Percentage
print "numOfPostingNeededToBePoppedAt70Percentage:",numOfPostingNeededToBePoppedAt70Percentage
print "numOfPostingNeededToBePoppedAt80Percentage:",numOfPostingNeededToBePoppedAt80Percentage
print "numOfPostingNeededToBePoppedAt90Percentage:",numOfPostingNeededToBePoppedAt90Percentage
print "numOfPostingNeededToBePoppedAt100Percentage:",numOfPostingNeededToBePoppedAt100Percentage

numOfOriginalTOP10PostingsRetainedInPrunedIndex = 0
# Updated by Wei on 2014/07/09, the computation of the following variable is just WRONG.
# numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex = 0
numOfBytes = 0
numOfPostingPopped = 0
numOfPostingBeInTOP10 = 0

# the current version
inputFileHandler = open(sys.argv[1],"rb")
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "--->posting file to evaluate:",sys.argv[1]
statinfo = os.stat(sys.argv[1])

fileSize = statinfo.st_size
print "file size:",fileSize
numOfResultsReturnedCountedUpTo10 = 0

while numOfBytes < fileSize:
    # current version
    # each time, read the info of 1M postings
    byteStringBuffer = inputFileHandler.read( 1000000 * 16)
    byteStringBufferIndexPosition = 0
    for i in range(0,1000000):
        byteString = byteStringBuffer[byteStringBufferIndexPosition:byteStringBufferIndexPosition+16]
        byteStringBufferIndexPosition += 16
        (termID,docID,currentProbability,impactScore) = unpack( "2I2f", byteString)
        postingKey = str(termID) + "_" + str(docID)
        
        if str(termID) not in termIDANDCurrentListLengthDict:
            termIDANDCurrentListLengthDict[str(termID)] = 1
        else:
            termIDANDCurrentListLengthDict[str(termID)] += 1
        
        if postingKey in top10PostingWithDocumentResultDict:
            # debug
            print "---TOP10PostingRetrieved-->",termID,docID,currentProbability,impactScore
            numOfOriginalTOP10PostingsRetainedInPrunedIndex += 1
            # numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex += len(top10PostingWithDocumentResultDict[postingKey])
        # debug
        # print "postingKey:",postingKey
        # print termID,docID,currentProbability,impactScore
        if postingKey in topKPostingWithDocumentResultDict:
            # interesting issue solved
            # print "touched."
            for currentDocumentResultKey in topKPostingWithDocumentResultDict[postingKey]:
                currentQID = currentDocumentResultKey.strip().split("_")[0]
                qIDsWithTopKDocumentResultImpactDict[currentQID][currentDocumentResultKey] += impactScore
                qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID][currentDocumentResultKey] += 1
                # tempCounter += 1

        if numOfPostingPopped == numOfPostingNeededToBePoppedAtDebugPercentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt1Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt2Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt3Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt4Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt5Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt6Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt7Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt8Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt9Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt10Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt15Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt20Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt30Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt40Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt50Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt60Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt70Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt80Percentage \
            or numOfPostingPopped == numOfPostingNeededToBePoppedAt90Percentage:
            
            numOfOriginalTOP10DocumentResultRetainedAtTOP10Position = 0
            numOfOriginalTOP10PostingRetainedAtTOP10Position = 0
            totalEvaluationCostFromPrunedIndexModel1 = 0
            totalEvaluationCostFromFullIndexModel1 = 0
            totalEvaluationCostFromPrunedIndexModel2 = 0
            totalEvaluationCostFromFullIndexModel2 = 0            
            currentQIDList = []
            currentQIDList = qIDsWithTopKDocumentResultImpactDict.keys()
            currentQIDList.sort(cmp=None, key=None, reverse=False)
            
            for currentQID in currentQIDList:
                currentQueryNumOfOriginalTOP10DocumentResultRetainedAtTOP10Position = 0
                # step1: compute the QPC for each query
                currentQueryQPCFromPrunedIndexModel1 = 0
                currentQueryQPCFromFullIndexModel1 = 0
                
                currentQueryQPCFromPrunedIndexModel2 = 0
                currentQueryQPCFromFullIndexModel2 = 0
                                
                shortestListFromPrunedIndex = 100000000
                longertListFromPrunedIndex = 0
                shortestListFromUnprunedIndex = 100000000
                longertListFromUnprunedIndex = 0
                for index,currentQueryTerm in enumerate(qIDWithQueryTermsDict[currentQID]): # loop through each query term in the query    
                    if currentQueryTerm in termANDTermIDDict:
                        currentQueryTermID = termANDTermIDDict[currentQueryTerm]
                        if currentQueryTermID in termIDANDCurrentListLengthDict:
                            currentQueryQPCFromPrunedIndexModel1 += termIDANDCurrentListLengthDict[currentQueryTermID]
                            if shortestListFromPrunedIndex > termIDANDCurrentListLengthDict[currentQueryTermID]:
                                shortestListFromPrunedIndex = termIDANDCurrentListLengthDict[currentQueryTermID]
                            if index == 1:
                                longertListFromPrunedIndex = termIDANDCurrentListLengthDict[currentQueryTermID]
                        if currentQueryTermID in termIDANDUnprunedListLengthDict:
                            currentQueryQPCFromFullIndexModel1 += termIDANDUnprunedListLengthDict[currentQueryTermID]
                            if shortestListFromUnprunedIndex > termIDANDUnprunedListLengthDict[currentQueryTermID]:
                                shortestListFromUnprunedIndex = termIDANDUnprunedListLengthDict[currentQueryTermID]
                            if index == 1:
                                longertListFromUnprunedIndex = termIDANDUnprunedListLengthDict[currentQueryTermID]
                #print "longertListFromPrunedIndex:",longertListFromPrunedIndex
                #print "shortestListFromPrunedIndex:",shortestListFromPrunedIndex
                #print "longertListFromUnprunedIndex:",longertListFromUnprunedIndex
                #print "shortestListFromUnprunedIndex:",shortestListFromUnprunedIndex
                
                if shortestListFromPrunedIndex == 100000000 or longertListFromPrunedIndex == 0:
                    currentQueryQPCFromPrunedIndexModel2 = 0
                    # debug
                    #print "pruned",currentQID,shortestListFromPrunedIndex,longertListFromPrunedIndex,currentQueryQPCFromPrunedIndexModel2
                else:
                    currentQueryQPCFromPrunedIndexModel2 = shortestListFromPrunedIndex * math.log(longertListFromPrunedIndex / shortestListFromPrunedIndex,2)
                    # debug
                    #print "pruned",currentQID,shortestListFromPrunedIndex,longertListFromPrunedIndex,currentQueryQPCFromPrunedIndexModel2
                    
                if shortestListFromUnprunedIndex == 100000000 or longertListFromUnprunedIndex == 0: 
                    currentQueryQPCFromFullIndexModel2 = 0
                    #print "pruned",currentQID,shortestListFromPrunedIndex,longertListFromPrunedIndex,currentQueryQPCFromPrunedIndexModel2
                else:
                    currentQueryQPCFromFullIndexModel2 = shortestListFromUnprunedIndex * math.log(longertListFromUnprunedIndex / shortestListFromUnprunedIndex,2)
                    # debug
                    #print "unpruned",currentQID,shortestListFromUnprunedIndex,longertListFromUnprunedIndex,currentQueryQPCFromFullIndexModel2
                    #print
                # accumulate each query to the total
                totalEvaluationCostFromPrunedIndexModel1 += currentQueryQPCFromPrunedIndexModel1
                totalEvaluationCostFromFullIndexModel1 += currentQueryQPCFromFullIndexModel1
                totalEvaluationCostFromPrunedIndexModel2 += currentQueryQPCFromPrunedIndexModel2
                totalEvaluationCostFromFullIndexModel2 += currentQueryQPCFromFullIndexModel2
                # debug
                #print currentQID,qIDWithQueryTermsDict[currentQID],currentQueryQPCFromPrunedIndexModel1,currentQueryQPCFromFullIndexModel1,currentQueryQPCFromPrunedIndexModel2,currentQueryQPCFromFullIndexModel2
                #print totalEvaluationCostFromPrunedIndexModel1,totalEvaluationCostFromFullIndexModel1
                #print totalEvaluationCostFromPrunedIndexModel2,totalEvaluationCostFromFullIndexModel2
                #print
                
                # step2: compute the quality for each query
                currentQIDDocumentResultList = []
                for currentDocumentResultKey in qIDsWithTopKDocumentResultImpactDict[currentQID]:
                    currentDocumentResultImpactScore = qIDsWithTopKDocumentResultImpactDict[currentQID][currentDocumentResultKey]
                    if currentDocumentResultImpactScore == 0.0:
                        pass
                    else:
                        currentQIDDocumentResultList.append( (currentDocumentResultKey,currentDocumentResultImpactScore) )
                
                currentQIDDocumentResultList.sort(cmp=None, key=itemgetter(1), reverse=True)
                for currentDocumentResultTuple in currentQIDDocumentResultList[:10]:
                    (thisDocumentResult,impactScore) = currentDocumentResultTuple
                    if thisDocumentResult in top10DocumentResultDict:
                        numOfOriginalTOP10DocumentResultRetainedAtTOP10Position += 1
                        currentQueryNumOfOriginalTOP10DocumentResultRetainedAtTOP10Position += 1
                        numOfOriginalTOP10PostingRetainedAtTOP10Position += qIDsWithTopKDocumentResultNumOfPostingsContributedDict[currentQID][thisDocumentResult]
                        thisDocID = thisDocumentResult.strip().split("_")[1]
                        print currentQID,thisDocID,impactScore,"NYU_IRTK"
                
                # step3: output the query processing cost for each query
                print "---fall through info-->:",currentQID,currentQueryNumOfOriginalTOP10DocumentResultRetainedAtTOP10Position,currentQueryQPCFromPrunedIndexModel1
            
            print "OVERALL:"
            print "len(qidsToEvaluateDict):",len(qidsToEvaluateDict)
            print "QUERY_LENGTH_TO_EVALUATE:",QUERY_LENGTH_TO_EVALUATE
            print "NUM_OF_TOPK_DOCUMENT_RESULT:",NUM_OF_TOPK_DOCUMENT_RESULT
            print "len(topKPostingWithDocumentResultDict):",len(topKPostingWithDocumentResultDict)
            print "len(top10DocumentResultDict):",len(top10DocumentResultDict)
            print "len(top10PostingWithDocumentResultDict):",len(top10PostingWithDocumentResultDict)
            print "NUM_OF_TOP_RESULTS_CUTOFF:",NUM_OF_TOP_RESULTS_CUTOFF
            print "NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:",NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE
            print "QUALITY aspect:"
            print "numOfOriginalTOP10DocumentResultRetainedAtTOP10Position:",numOfOriginalTOP10DocumentResultRetainedAtTOP10Position
            print "numOfOriginalTOP10PostingRetainedAtTOP10Position:",numOfOriginalTOP10PostingRetainedAtTOP10Position
            # print "numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex:",numOfOriginalTOP10DocumentResultsRetainedInPrunedIndex
            print "numOfOriginalTOP10PostingsRetainedInPrunedIndex:",numOfOriginalTOP10PostingsRetainedInPrunedIndex
            print "PERFORMANCE aspect:"
            print "totalEvaluationCostFromPrunedIndexModel1:",totalEvaluationCostFromPrunedIndexModel1
            print "totalEvaluationCostFromFullIndexModel1:",totalEvaluationCostFromFullIndexModel1
            print "totalEvaluationCostFromPrunedIndexModel2:",totalEvaluationCostFromPrunedIndexModel2 
            print "totalEvaluationCostFromFullIndexModel2:",totalEvaluationCostFromFullIndexModel2
            print "numOfPostingPopped:",numOfPostingPopped
        numOfBytes += 12
        numOfPostingPopped += 1
        if numOfPostingPopped % 1000000 == 0:
            print str(numOfPostingPopped),"postings have been examined."

inputFileHandler.close()
print "inputFileName0:",inputFileName0
print "inputFileName1:",inputFileName1
print "program ends."
